package cn.db101.fairy.crawl.processor

import us.codecraft.webmagic.Page
import us.codecraft.webmagic.Site
import us.codecraft.webmagic.processor.PageProcessor
import us.codecraft.webmagic.selector.Html

class HzProcessor(val categoryId: Int): PageProcessor {

    override fun process(page: Page?) {

        // 获取文章列表
        if (page?.url?.get()?.contains("page") == true) {
            page?.html?.xpath("//h2[@class='item-title']/a")?.all()?.forEach {
                val href = Html(it).xpath("//a/@href").get()
                page.addTargetRequest(href)
            }

            page.setSkip(true)
        } else {
            // 获取文章内容
            val content = mutableListOf<String>()

            page?.html?.xpath("//div[@class='entry-content']/p/text()")?.all()?.forEach {
                content.add("<p>${it}</p>")
            }

            page?.putField("content", content.joinToString(separator = ""))

            // 获取文章标题
            page?.html?.xpath("//h1[@class='entry-title']/text()")?.all()?.forEach {
                page.putField("name", it)
            }

            page?.putField("categoryId", categoryId)

        }

    }

    override fun getSite(): Site {

        return Site.me().setRetryTimes(2).setSleepTime(1000)
    }
}